{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import pydicom\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/mnt/DATA/rsna/\"\n",
    "meta_data_dir = data_dir + \"gzip/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped_cols = ['BitsAllocated', 'BitsStored',\n",
    "       'Columns', 'HighBit', 'ImageOrientationPatient_0',\n",
    "       'ImageOrientationPatient_1', 'ImageOrientationPatient_2',\n",
    "       'ImageOrientationPatient_3', 'ImageOrientationPatient_4',\n",
    "       'ImageOrientationPatient_5', 'ImagePositionPatient_0',\n",
    "       'ImagePositionPatient_1', 'Modality',\n",
    "       'PhotometricInterpretation', 'PixelRepresentation',\n",
    "       'PixelSpacing_0', 'PixelSpacing_1', 'RescaleIntercept', 'RescaleSlope',\n",
    "       'Rows', 'SOPInstanceUID', 'SamplesPerPixel', 'SeriesInstanceUID',\n",
    "       'StudyID',]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the labels & metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metadata(image_dir):\n",
    "\n",
    "    labels = [\n",
    "        'BitsAllocated', 'BitsStored', 'Columns', 'HighBit', \n",
    "        'ImageOrientationPatient_0', 'ImageOrientationPatient_1', 'ImageOrientationPatient_2',\n",
    "        'ImageOrientationPatient_3', 'ImageOrientationPatient_4', 'ImageOrientationPatient_5',\n",
    "        'ImagePositionPatient_0', 'ImagePositionPatient_1', 'ImagePositionPatient_2',\n",
    "        'Modality', 'PatientID', 'PhotometricInterpretation', 'PixelRepresentation',\n",
    "        'PixelSpacing_0', 'PixelSpacing_1', 'RescaleIntercept', 'RescaleSlope', 'Rows', 'SOPInstanceUID',\n",
    "        'SamplesPerPixel', 'SeriesInstanceUID', 'StudyID', 'StudyInstanceUID', \n",
    "        'WindowCenter', 'WindowWidth', 'Image',\n",
    "    ]\n",
    "\n",
    "    data = {l: [] for l in labels}\n",
    "\n",
    "    for image in tqdm_notebook(os.listdir(image_dir)):\n",
    "        data[\"Image\"].append(image[:-4])\n",
    "\n",
    "        ds = pydicom.dcmread(os.path.join(image_dir, image))\n",
    "\n",
    "        for metadata in ds.dir():\n",
    "            if metadata != \"PixelData\":\n",
    "                metadata_values = getattr(ds, metadata)\n",
    "                if type(metadata_values) == pydicom.multival.MultiValue and metadata not in [\"WindowCenter\", \"WindowWidth\"]:\n",
    "                    for i, v in enumerate(metadata_values):\n",
    "                        data[f\"{metadata}_{i}\"].append(v)\n",
    "                else:\n",
    "                    if type(metadata_values) == pydicom.multival.MultiValue and metadata in [\"WindowCenter\", \"WindowWidth\"]:\n",
    "                        data[metadata].append(metadata_values[0])\n",
    "                    else:\n",
    "                        data[metadata].append(metadata_values)\n",
    "\n",
    "    return pd.DataFrame(data).set_index(\"Image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate metadata dataframes\n",
    "train_metadata = get_metadata(os.path.join(data_dir, \"stage_1_train_images\"))\n",
    "test_metadata = get_metadata(os.path.join(data_dir, \"stage_1_test_images\"))\n",
    "\n",
    "train_metadata.to_parquet(f'{meta_data_dir}/train_metadata.parquet.gzip', compression='gzip')\n",
    "test_metadata.to_parquet(f'{meta_data_dir}/test_metadata.parquet.gzip', compression='gzip')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Note: 'ImagePositionPatient_2' is z-axis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(data_dir + \"stage_1_train.csv\").drop_duplicates()\n",
    "train_df['image'] = train_df[\"ID\"].str.slice(stop=12)\n",
    "train_df['Diagnosis'] = train_df['ID'].str.slice(start=13)\n",
    "train_labels = train_df.pivot(index=\"image\", \n",
    "                              columns=\"Diagnosis\", \n",
    "                              values=\"Label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(data_dir + \"stage_1_sample_submission.csv\").drop_duplicates()\n",
    "test_df['image'] = test_df[\"ID\"].str.slice(stop=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train_labels[\"any\"],\n",
    "                   train_labels[\"epidural\"], train_labels[\"intraparenchymal\"],\n",
    "                   train_labels[\"intraventricular\"], train_labels[\"subarachnoid\"],\n",
    "                   train_labels[\"subdural\"]], 1).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test_df[\"image\"].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_parq = pd.read_parquet(meta_data_dir + \"train_metadata.parquet.gzip\")\n",
    "train_parq[\"image\"] = train_parq.index\n",
    "test_parq = pd.read_parquet(meta_data_dir + \"test_metadata.parquet.gzip\")\n",
    "test_parq[\"image\"] = test_parq.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_train = pd.merge(train, train_parq, how=\"inner\", on=\"image\")\n",
    "merged_test = pd.merge(test, test_parq, how=\"inner\", on=\"image\")\n",
    "merged_train.drop(columns=dropped_cols, inplace=True)\n",
    "merged_test.drop(columns=dropped_cols, inplace=True)\n",
    "\n",
    "del train, test, train_parq, test_parq\n",
    "gc.collect()\n",
    "\n",
    "merged_train = merged_train.groupby([\"StudyInstanceUID\"]) \\\n",
    "    .apply(lambda x: x.sort_values([\"ImagePositionPatient_2\"], \n",
    "                                   ascending = True))\\\n",
    "    .reset_index(drop=True)\n",
    "\n",
    "merged_test = merged_test.groupby([\"StudyInstanceUID\"]) \\\n",
    "    .apply(lambda x: x.sort_values([\"ImagePositionPatient_2\"], \n",
    "                                   ascending = True)) \\\n",
    "    .reset_index(drop=True)\n",
    "\n",
    "# merged_train.to_csv(data_dir + \"train_metadata.csv\", index=False)\n",
    "# merged_test.to_csv(data_dir + \"test_metadata.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>any</th>\n",
       "      <th>epidural</th>\n",
       "      <th>intraparenchymal</th>\n",
       "      <th>intraventricular</th>\n",
       "      <th>subarachnoid</th>\n",
       "      <th>subdural</th>\n",
       "      <th>ImagePositionPatient_2</th>\n",
       "      <th>PatientID</th>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <th>WindowCenter</th>\n",
       "      <th>WindowWidth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>ID_3a422b8d7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28.2225</td>\n",
       "      <td>ID_e0d2de32</td>\n",
       "      <td>ID_00047d6503</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>ID_490b10d5a</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>33.2225</td>\n",
       "      <td>ID_e0d2de32</td>\n",
       "      <td>ID_00047d6503</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>ID_be2a0ca1c</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38.2225</td>\n",
       "      <td>ID_e0d2de32</td>\n",
       "      <td>ID_00047d6503</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>ID_af42e31f3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>43.2225</td>\n",
       "      <td>ID_e0d2de32</td>\n",
       "      <td>ID_00047d6503</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>ID_3131664ab</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48.2225</td>\n",
       "      <td>ID_e0d2de32</td>\n",
       "      <td>ID_00047d6503</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>674253</td>\n",
       "      <td>ID_7b0c5edc0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1310.0500</td>\n",
       "      <td>ID_822276a2</td>\n",
       "      <td>ID_fffdba8d7b</td>\n",
       "      <td>40.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>674254</td>\n",
       "      <td>ID_e8e195f90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1315.0500</td>\n",
       "      <td>ID_822276a2</td>\n",
       "      <td>ID_fffdba8d7b</td>\n",
       "      <td>40.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>674255</td>\n",
       "      <td>ID_d64ef9ea6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1320.0500</td>\n",
       "      <td>ID_822276a2</td>\n",
       "      <td>ID_fffdba8d7b</td>\n",
       "      <td>40.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>674256</td>\n",
       "      <td>ID_5838a09b1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1325.0500</td>\n",
       "      <td>ID_822276a2</td>\n",
       "      <td>ID_fffdba8d7b</td>\n",
       "      <td>40.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>674257</td>\n",
       "      <td>ID_83d5723ef</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1330.0500</td>\n",
       "      <td>ID_822276a2</td>\n",
       "      <td>ID_fffdba8d7b</td>\n",
       "      <td>40.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>674258 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               image  any  epidural  intraparenchymal  intraventricular  \\\n",
       "0       ID_3a422b8d7    0         0                 0                 0   \n",
       "1       ID_490b10d5a    0         0                 0                 0   \n",
       "2       ID_be2a0ca1c    0         0                 0                 0   \n",
       "3       ID_af42e31f3    0         0                 0                 0   \n",
       "4       ID_3131664ab    0         0                 0                 0   \n",
       "...              ...  ...       ...               ...               ...   \n",
       "674253  ID_7b0c5edc0    0         0                 0                 0   \n",
       "674254  ID_e8e195f90    0         0                 0                 0   \n",
       "674255  ID_d64ef9ea6    0         0                 0                 0   \n",
       "674256  ID_5838a09b1    0         0                 0                 0   \n",
       "674257  ID_83d5723ef    0         0                 0                 0   \n",
       "\n",
       "        subarachnoid  subdural  ImagePositionPatient_2    PatientID  \\\n",
       "0                  0         0                 28.2225  ID_e0d2de32   \n",
       "1                  0         0                 33.2225  ID_e0d2de32   \n",
       "2                  0         0                 38.2225  ID_e0d2de32   \n",
       "3                  0         0                 43.2225  ID_e0d2de32   \n",
       "4                  0         0                 48.2225  ID_e0d2de32   \n",
       "...              ...       ...                     ...          ...   \n",
       "674253             0         0               1310.0500  ID_822276a2   \n",
       "674254             0         0               1315.0500  ID_822276a2   \n",
       "674255             0         0               1320.0500  ID_822276a2   \n",
       "674256             0         0               1325.0500  ID_822276a2   \n",
       "674257             0         0               1330.0500  ID_822276a2   \n",
       "\n",
       "       StudyInstanceUID  WindowCenter  WindowWidth  \n",
       "0         ID_00047d6503          30.0         80.0  \n",
       "1         ID_00047d6503          30.0         80.0  \n",
       "2         ID_00047d6503          30.0         80.0  \n",
       "3         ID_00047d6503          30.0         80.0  \n",
       "4         ID_00047d6503          30.0         80.0  \n",
       "...                 ...           ...          ...  \n",
       "674253    ID_fffdba8d7b          40.0         80.0  \n",
       "674254    ID_fffdba8d7b          40.0         80.0  \n",
       "674255    ID_fffdba8d7b          40.0         80.0  \n",
       "674256    ID_fffdba8d7b          40.0         80.0  \n",
       "674257    ID_fffdba8d7b          40.0         80.0  \n",
       "\n",
       "[674258 rows x 12 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split by StudyInstanceUID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "study_id_counts = merged_train.groupby([\"StudyInstanceUID\"])[\"any\"].sum()\n",
    "study_ids, slice_counts = study_id_counts.index.values, study_id_counts.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarized_slice_counts = slice_counts > 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, random_state=2709)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = {}\n",
    "for i, (train_idx, val_idx) in enumerate(skf.split(study_ids, binarized_slice_counts)):\n",
    "    folds[i] = (study_ids[train_idx], study_ids[val_idx])\n",
    "\n",
    "    np.save(data_dir + \"train_fold\" + str(i) + \".npy\", study_ids[train_idx])\n",
    "    np.save(data_dir + \"valid_fold\" + str(i) + \".npy\", study_ids[val_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fold in range(5):\n",
    "    print(\"\\n================FOLD {}================\".format(str(fold)))\n",
    "    \n",
    "    for col in [\n",
    "        \"any\", \n",
    "        \"intraparenchymal\", \"intraventricular\", \n",
    "        \"subarachnoid\", \"subdural\", \"epidural\"\n",
    "    ]:\n",
    "        train_df = merged_train[merged_train[\"StudyInstanceUID\"].isin(folds[fold][0])]\n",
    "        num_train_strat_sid = train_df[col].sum()\n",
    "        print(col,\n",
    "              \"train samples: {:.1f} ===\".format(num_train_strat_sid),\n",
    "              \"alpha: {:.3f}\".format(compute_alpha(\n",
    "                  num_train_strat_sid / train_df['epidural'].sum())\n",
    "                                    )\n",
    "             )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split by PatientID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_id_counts = merged_train.groupby([\"PatientID\"])[\"any\"].sum()\n",
    "patient_ids, slice_counts = patient_id_counts.index.values, patient_id_counts.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "slice_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarized_slice_counts = slice_counts > 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarized_slice_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(binarized_slice_counts) / 17079"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, random_state=2709)\n",
    "\n",
    "folds = {}\n",
    "for i, (train_idx, val_idx) in enumerate(skf.split(patient_ids, binarized_slice_counts)):\n",
    "    folds[i] = (patient_ids[train_idx], patient_ids[val_idx])\n",
    "    np.save(data_dir + \"train_patients_fold\" + str(i) + \".npy\", patient_ids[train_idx])\n",
    "    np.save(data_dir + \"valid_patients_fold\" + str(i) + \".npy\", patient_ids[val_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fold in range(5):\n",
    "    print(\"\\n================FOLD {}================\".format(str(fold)))\n",
    "    \n",
    "    for col in [\n",
    "        \"any\", \n",
    "        \"intraparenchymal\", \"intraventricular\", \n",
    "        \"subarachnoid\", \"subdural\", \"epidural\"\n",
    "    ]:\n",
    "        train_df = merged_train[merged_train[\"PatientID\"].isin(folds[fold][0])]\n",
    "        num_train_strat_sid = train_df[col].sum()\n",
    "        print(col,\n",
    "              \"train samples: {:.1f} ===\".format(num_train_strat_sid),\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(285,)\n",
      "(17079,)\n",
      "(2144,)\n"
     ]
    }
   ],
   "source": [
    "train_meta = pd.read_csv(\"/mnt/DATA/rsna/train_metadata.csv\")\n",
    "test_meta = pd.read_csv(\"/mnt/DATA/rsna/test_metadata.csv\")\n",
    "# check patient id overlap\n",
    "print(np.intersect1d(train_meta[\"PatientID\"].unique(),\n",
    "               test_meta[\"PatientID\"].unique()).shape)\n",
    "# check train patient id\n",
    "print(train_meta[\"PatientID\"].unique().shape)\n",
    "# check test patient id\n",
    "print(test_meta[\"PatientID\"].unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "overlap_patient_ids = np.intersect1d(train_meta[\"PatientID\"].unique(),\n",
    "                                     test_meta[\"PatientID\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>ImagePositionPatient_2</th>\n",
       "      <th>PatientID</th>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <th>WindowCenter</th>\n",
       "      <th>WindowWidth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>306</td>\n",
       "      <td>ID_38eff6d7a</td>\n",
       "      <td>43.957554</td>\n",
       "      <td>ID_10c07909</td>\n",
       "      <td>ID_0136ebaa38</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>307</td>\n",
       "      <td>ID_2c0775e47</td>\n",
       "      <td>46.571781</td>\n",
       "      <td>ID_10c07909</td>\n",
       "      <td>ID_0136ebaa38</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>308</td>\n",
       "      <td>ID_c172014ce</td>\n",
       "      <td>49.186012</td>\n",
       "      <td>ID_10c07909</td>\n",
       "      <td>ID_0136ebaa38</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>309</td>\n",
       "      <td>ID_942888df3</td>\n",
       "      <td>51.800240</td>\n",
       "      <td>ID_10c07909</td>\n",
       "      <td>ID_0136ebaa38</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>310</td>\n",
       "      <td>ID_4c7153a14</td>\n",
       "      <td>54.412552</td>\n",
       "      <td>ID_10c07909</td>\n",
       "      <td>ID_0136ebaa38</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78436</td>\n",
       "      <td>ID_fb7d67d85</td>\n",
       "      <td>113.664000</td>\n",
       "      <td>ID_4f7414e4</td>\n",
       "      <td>ID_ffb2e70ba3</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78437</td>\n",
       "      <td>ID_2232eadb1</td>\n",
       "      <td>118.733000</td>\n",
       "      <td>ID_4f7414e4</td>\n",
       "      <td>ID_ffb2e70ba3</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78438</td>\n",
       "      <td>ID_dcf40f4d7</td>\n",
       "      <td>123.803000</td>\n",
       "      <td>ID_4f7414e4</td>\n",
       "      <td>ID_ffb2e70ba3</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78439</td>\n",
       "      <td>ID_8878be83a</td>\n",
       "      <td>128.872000</td>\n",
       "      <td>ID_4f7414e4</td>\n",
       "      <td>ID_ffb2e70ba3</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78440</td>\n",
       "      <td>ID_f7fbd4e0c</td>\n",
       "      <td>133.942000</td>\n",
       "      <td>ID_4f7414e4</td>\n",
       "      <td>ID_ffb2e70ba3</td>\n",
       "      <td>30.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11651 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              image  ImagePositionPatient_2    PatientID StudyInstanceUID  \\\n",
       "306    ID_38eff6d7a               43.957554  ID_10c07909    ID_0136ebaa38   \n",
       "307    ID_2c0775e47               46.571781  ID_10c07909    ID_0136ebaa38   \n",
       "308    ID_c172014ce               49.186012  ID_10c07909    ID_0136ebaa38   \n",
       "309    ID_942888df3               51.800240  ID_10c07909    ID_0136ebaa38   \n",
       "310    ID_4c7153a14               54.412552  ID_10c07909    ID_0136ebaa38   \n",
       "...             ...                     ...          ...              ...   \n",
       "78436  ID_fb7d67d85              113.664000  ID_4f7414e4    ID_ffb2e70ba3   \n",
       "78437  ID_2232eadb1              118.733000  ID_4f7414e4    ID_ffb2e70ba3   \n",
       "78438  ID_dcf40f4d7              123.803000  ID_4f7414e4    ID_ffb2e70ba3   \n",
       "78439  ID_8878be83a              128.872000  ID_4f7414e4    ID_ffb2e70ba3   \n",
       "78440  ID_f7fbd4e0c              133.942000  ID_4f7414e4    ID_ffb2e70ba3   \n",
       "\n",
       "       WindowCenter  WindowWidth  \n",
       "306            30.0         80.0  \n",
       "307            30.0         80.0  \n",
       "308            30.0         80.0  \n",
       "309            30.0         80.0  \n",
       "310            30.0         80.0  \n",
       "...             ...          ...  \n",
       "78436          30.0         80.0  \n",
       "78437          30.0         80.0  \n",
       "78438          30.0         80.0  \n",
       "78439          30.0         80.0  \n",
       "78440          30.0         80.0  \n",
       "\n",
       "[11651 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_meta[test_meta[\"PatientID\"].isin(overlap_patient_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
